Ceph分离部署

场景说明

管理、业务、存储接入交换机独立部署、物理隔离，便于扩容服务器。管理和业务网络共用核心交换机，由核心交换机提供管理网络和业务网络网关。同位置交换机两两组建堆叠或者M-LAG，每台服务器使用6个网口（跨网卡组合成3对）分别用作管理、业务和存储，每台存储节点服务器使用4个网口（跨网卡组合成2对）分别作为管理和存储，每对网口配置bond分别上联两台接入交换机。

场景拓扑

image-20220712114441303

连接示例

  • 管理交换机端口互连表

image-20220809133925168

  • 业务交换机端口互连表

image-20220809133956857

  • 存储交换机端口互连表

image-20220809134021180

网络规划

请根据服务器数量、云主机数量规划IP&VLAN&VXLAN，建议预留冗余量便于扩容。

  • VLAN规划

    • 管理：2-89
    • 存储：90-99
    • 私有：100-1999（扁平网络+VPC网络）
    • 公有：2000-2999

  • VXLAN规划

    • VXLAN Pool：5000-5999

  • IP规划

    • 管理网络IP：192.168.248.0/22（4个C）
    • 公有网络IP：172.31.0.0/20（16个C）
    • 私有网络IP：10.255.0.0/16（256个C）

  • 生成树规划

    • 此网络作为独立二层网络时建议把Spine配置为根桥；如果此网络二层接入现网，则需与网络运维工程师共同规划根桥，不可私自指定根桥，避免发生生成树抢根导致网络中断。
    • 一般情况下建议所有与服务器互连的交换机端口配置成边缘端口，可以加速生成树收敛、提高网络稳定性。

配置举例

下面以服务器服务器bond-lacp、交换机M-LAG为例提供服务器网络配置和交换机配置参考。

  • 服务器管理网络配置

     zs-bond-ab -c bond0                //创建管理网卡聚合bond0
 zs-nic-to-bond -a bond0 em1        //将管理网卡em1绑定到bond0上
 zs-nic-to-bond -a bond0 em5        //将管理网卡em5绑定到bond0上，建议同一个bond中网卡型号相同
 zs-network-setting -b bond0 192.168.248.11 255.255.255.0 192.168.248.1    //配置管理IP、掩码和网关

  • 服务器业务网络配置

     zs-bond-ab -c bond1                //创建管理网卡聚合bond1
 zs-nic-to-bond -a bond1 em2        //将管理网卡em2绑定到bond1上
 zs-nic-to-bond -a bond1 em6        //将管理网卡em6绑定到bond1上，建议同一个bond中网卡型号相同

  • 服务器存储网络配置

     zs-bond-ab -c bond2                //创建管理网卡聚合bond0
 zs-nic-to-bond -a bond0 em3        //将管理网卡em3绑定到bond2上
 zs-nic-to-bond -a bond0 em7        //将管理网卡em7绑定到bond2上，建议同一个bond中网卡型号相同
 zs-network-setting -b bond2 192.168.252.11 255.255.255.0    //配置管理IP、掩码

  • 交换机配置（以华为交换机为例，仅列出关键配置）

     ############## 核心交换机Spine1A（华为CE12804） ################

 #配置交换机名称
 sysname Spine1A    
 #配置M-LAG dfs-group
 dfs-group 1
  priority 200
  source ip X.X.X.X
 #配置M-LAG peer-link
 interface Eth-Trunk0
  mode lacp-static
  trunkport 100GE 1/0/35
  trunkport 100GE 2/0/35
  peer-link 1
 #配置生成树
 stp root primary
 stp tc-protection
 stp bpdu-protection
 stp mode rstp
 stp v-stp enable
 #配置VRF，用于隔离管理和业务网络
 #
 ip vpn-instance mgmt
  ipv4-family
   route-distinguisher 1:1
 #
 ip vpn-instance biz
  ipv4-family
   route-distinguisher 1:2
 #配置管理
 vlan 2            
  description == Server-Management ==
 #
 interface Vlanif2
  ip binding vpn-instance mgmt
  ip address 192.168.248.254 24
  mac-address 0000-5e00-01ff
 #配置业务VLAN
 vlan batch 100 101 2000 2001
 #
 vlan 100
  description == Private-1 ==
 #
 vlan 101
  description == Private-2 ==
 #
 vlan 2000
  description == Public-1 ==
 #
 vlan 2001
  description == Public-2 ==
 #
 interface Vlanif2000
  ip binding vpn-instance biz
  ip address 172.31.0.254 24
  mac-address 0000-5e00-01ff
 #
 interface Vlanif2001
  ip binding vpn-instance biz
  ip address 172.31.1.254 24
  mac-address 0000-5e00-01ff
 #配置下联管理接入交换机Leaf1端口
 interface Eth-Trunk1
  description == To_Leaf1 ==
  mode lacp-static
  trunkport 100GE 1/0/1
  trunkport 100GE 2/0/1
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 2
 #
 #配置下联业务接入交换机Leaf2端口
 interface Eth-Trunk2
  description == To_Leaf2 ==
  mode lacp-static
  trunkport 100GE 1/0/2
  trunkport 100GE 2/0/2
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 100 101 2000 2001
 #

 ############## 核心交换机Spine1B（华为CE12804） ################
 #配置交换机名称
 sysname Spine1B
 #配置M-LAG dfs-group
 dfs-group 1
  priority 100
  source ip X.X.X.X
 #配置M-LAG peer-link
 interface Eth-Trunk0
  mode lacp-static
  trunkport 100GE 1/0/35
  trunkport 100GE 2/0/35
  peer-link 1
 #配置生成树
 stp root secondary
 stp tc-protection
 stp bpdu-protection
 stp mode rstp
 stp v-stp enable
 #
 #其他配置请参考核心交换机Spine1A

 ############## 管理接入交换机Leaf1A（华为CE6881-48S6CQ） ################
 #配置交换机名称
 sysname Leaf1A
 #配置M-LAG dfs-group
 dfs-group 1
  priority 200
  source ip X.X.X.X
 #配置M-LAG peer-link
 interface Eth-Trunk0
  mode lacp-static
  trunkport 100GE 1/0/4
  trunkport 100GE 1/0/6
  peer-link 1
 #配置生成树
 stp tc-protection
 stp bpdu-protection
 stp mode rstp
 stp v-stp enable
 #配置管理VLAN
 vlan 2            
  description == Server-Management ==
 #配置上联核心Spine1端口
 interface Eth-Trunk100
  description == To_Spine1 ==
  mode lacp-static
  trunkport 100GE 1/0/1
  trunkport 100GE 1/0/3
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 2
  dfs-group 1 m-lag 100
 #配置下联服务器zstack-1管理端口
 interface Eth-Trunk1
  description == To_zstack-1 ==
  mode lacp-static
  trunkport 10GE 1/0/1
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 2
  port trunk pvid vlan 2
  stp edged-port enable
  dfs-group 1 m-lag 1
 #配置下联服务器zstack-2管理端口
 interface Eth-Trunk2
  description == To_zstack-2 ==
  mode lacp-static
  trunkport 10GE 1/0/2
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 2
  port trunk pvid vlan 2
  stp edged-port enable
  dfs-group 1 m-lag 2
 #配置下联服务器zstack-3管理端口
 interface Eth-Trunk3
  description == To_zstack-3 ==
  mode lacp-static
  trunkport 10GE 1/0/3
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 2
  port trunk pvid vlan 2
  stp edged-port enable
  dfs-group 1 m-lag 3
 #配置下联服务器zstone-1管理端口
 interface Eth-Trunk4
  description == To_zstone-1 ==
  mode lacp-static
  trunkport 10GE 1/0/4
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 2
  port trunk pvid vlan 2
  stp edged-port enable
  dfs-group 1 m-lag 4
 #配置下联服务器zstone-2管理端口
 interface Eth-Trunk5
  description == To_zstone-2 ==
  mode lacp-static
  trunkport 10GE 1/0/5
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 2
  port trunk pvid vlan 2
  stp edged-port enable
  dfs-group 1 m-lag 5
 #配置下联服务器zstone-3管理端口
 interface Eth-Trunk6
  description == To_zstone-3 ==
  mode lacp-static
  trunkport 10GE 1/0/6
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 2
  port trunk pvid vlan 2
  stp edged-port enable
  dfs-group 1 m-lag 6
 #

 ############## 管理接入交换机Leaf1B（华为CE6881-48S6CQ） ################
 #配置交换机名称
 sysname Leaf1B
 #配置M-LAG dfs-group
 dfs-group 1
  priority 100
  source ip X.X.X.X
 #配置M-LAG peer-link
 interface Eth-Trunk0
  mode lacp-static
  trunkport 100GE 1/0/4
  trunkport 100GE 1/0/6
  peer-link 1
 #
 #其他配置请参考管理接入交换机Leaf1A

 ############## 业务接入交换机Leaf2A（华为CE6881-48S6CQ） ################
 #配置交换机名称
 sysname Leaf2A
 #配置M-LAG dfs-group
 dfs-group 1
  priority 200
  source ip X.X.X.X
 #配置M-LAG peer-link
 interface Eth-Trunk0
  mode lacp-static
  trunkport 100GE 1/0/4
  trunkport 100GE 1/0/6
  peer-link 1
 #配置生成树
 stp tc-protection
 stp bpdu-protection
 stp mode rstp
 stp v-stp enable
 #配置业务VLAN
 vlan batch 100 101 2000 2001
 #
 vlan 100
  description == Private-1 ==
 #
 vlan 101
  description == Private-2 ==
 #
 vlan 2000
  description == Public-1 ==
 #
 vlan 2001
  description == Public-2 ==
 #配置上联核心Spine1端口
 interface Eth-Trunk100
  description == To_Spine1 ==
  mode lacp-static
  trunkport 100GE 1/0/1
  trunkport 100GE 1/0/3
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 100 101 2000 2001
  dfs-group 1 m-lag 100
 #配置下联服务器zstack-1业务端口
 interface Eth-Trunk1
  description == To_zstack-1 ==
  mode lacp-static
  trunkport 10GE 1/0/1
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 100 101 2000 2001
  stp edged-port enable
  dfs-group 1 m-lag 1
 #配置下联服务器zstack-2业务端口
 interface Eth-Trunk2
  description == To_zstack-2 ==
  mode lacp-static
  trunkport 10GE 1/0/2
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 100 101 2000 2001
  stp edged-port enable
  dfs-group 1 m-lag 2
 #配置下联服务器zstack-3业务端口
 interface Eth-Trunk3
  description == To_zstack-3 ==
  mode lacp-static
  trunkport 10GE 1/0/3
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 100 101 2000 2001
  stp edged-port enable
  dfs-group 1 m-lag 3
 #

 ############## 业务接入交换机Leaf2B（华为CE6881-48S6CQ） ################
 #配置交换机名称
 sysname Leaf2B
 #配置M-LAG dfs-group
 dfs-group 1
  priority 100
  source ip X.X.X.X
 #配置M-LAG peer-link
 interface Eth-Trunk0
  mode lacp-static
  trunkport 100GE 1/0/4
  trunkport 100GE 1/0/6
  peer-link 1
 #
 #其他配置请参考业务接入交换机Leaf2A

 ############## 存储接入交换机Leaf3A（华为CE6881-48S6CQ） ################
 #配置交换机名称
 sysname Leaf3A
 #配置M-LAG dfs-group
 dfs-group 1
  priority 200
  source ip X.X.X.X
 #配置M-LAG peer-link
 interface Eth-Trunk0
  mode lacp-static
  trunkport 100GE 1/0/4
  trunkport 100GE 1/0/6
  peer-link 1
 #配置生成树
 stp tc-protection
 stp bpdu-protection
 stp mode rstp
 stp v-stp enable
 #配置管理VLAN
 vlan 50            
  description == Storage ==
 #配置下联服务器zstack-1存储端口
 interface Eth-Trunk1
  description == To_zstack-1 ==
  mode lacp-static
  trunkport 10GE 1/0/1
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 50
  port trunk pvid vlan 50
  stp edged-port enable
  dfs-group 1 m-lag 1
 #配置下联服务器zstack-2存储端口
 interface Eth-Trunk2
  description == To_zstack-2 ==
  mode lacp-static
  trunkport 10GE 1/0/2
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 50
  port trunk pvid vlan 50
  stp edged-port enable
  dfs-group 1 m-lag 2
 #配置下联服务器zstack-3存储端口
 interface Eth-Trunk3
  description == To_zstack-3 ==
  mode lacp-static
  trunkport 10GE 1/0/3
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 50
  port trunk pvid vlan 50
  stp edged-port enable
  dfs-group 1 m-lag 3
 #配置下联服务器zstone-1存储端口
 interface Eth-Trunk4
  description == To_zstone-1 ==
  mode lacp-static
  trunkport 10GE 1/0/4
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 50
  port trunk pvid vlan 50
  stp edged-port enable
  dfs-group 1 m-lag 4
 #配置下联服务器zstone-2存储端口
 interface Eth-Trunk5
  description == To_zstone-2 ==
  mode lacp-static
  trunkport 10GE 1/0/5
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 50
  port trunk pvid vlan 50
  stp edged-port enable
  dfs-group 1 m-lag 5
 #配置下联服务器zstone-3存储端口
 interface Eth-Trunk6
  description == To_zstone-3 ==
  mode lacp-static
  trunkport 10GE 1/0/6
  port link-type trunk
  undo port trunk allow-pass vlan 1
  port trunk allow-pass vlan 50
  port trunk pvid vlan 50
  stp edged-port enable
  dfs-group 1 m-lag 6
  #

 ############## 存储接入交换机Leaf3B（华为CE6881-48S6CQ） ################
 #配置交换机名称
 sysname Leaf3B
 #配置M-LAG dfs-group
 dfs-group 1
  priority 100
  source ip X.X.X.X
 #配置M-LAG peer-link
 interface Eth-Trunk0
  mode lacp-static
  trunkport 100GE 1/0/4
  trunkport 100GE 1/0/6
  peer-link 1
 #
 #其他配置请参考存储接入交换机Leaf3A

results matching ""

    No results matching ""